Pre-process on Client Train Images¶
from PIL import Image
from IPython.display import display
from sklearn.model_selection import train_test_split
from matplotlib import pyplot as plt
import pylab, random, os, math, sklearn
import numpy as np
import tensorflow as tf
# To cancel scientific notation
np.set_printoptions(suppress=True)
%matplotlib inline
# %matplotlib tk
%config InlineBackend.figure_format = 'retina'
# To set plotting style
plt.style.use("classic")
plt.style.use("bmh")
plt.style.use("seaborn-whitegrid")
# To hide warnings
import warnings
warnings.filterwarnings("ignore")
1. Extract Labels from XML Files¶
PATH = "/Users/ray/Desktop/crack-images/LCMS-Range-Labeled"
dirs = os.listdir(PATH)
dirs
ANNO_PATH = os.path.join(PATH, "annotations")
ANNO_PATH
IMG_PATH = os.path.join(PATH, "images")
IMG_PATH
1.1 A Trial on a Single Image¶
Let's have look at a sample XML file:
xml_list = os.listdir(ANNO_PATH)
sample_xml = xml_list[0]
with open(os.path.join(ANNO_PATH, sample_xml)) as file:
print(file.read())
Use beautiful Soup to parse the file:
Number of cracks:
from bs4 import BeautifulSoup
with open(os.path.join(ANNO_PATH, sample_xml)) as file:
soup = BeautifulSoup(file.read(), 'xml')
len(soup.find_all("object"))
Name of the image file:
filename = soup.filename.text
filename
Have a look at the image:
from tensorflow.keras.preprocessing.image import load_img
from tensorflow.keras.preprocessing.image import img_to_array
img = img_to_array(load_img(os.path.join(IMG_PATH, filename))).astype(int)
plt.figure(figsize=(10,20))
plt.imshow(img)
plt.show()
Image size:
img.shape
Crack position and crack type:
for crack in soup.find_all("object"):
name = crack.find("name").text
box = crack.bndbox
print(f"Type: {name}, xmin: {box.xmin.text}, xmax: {box.xmax.text}, ymin: {box.ymin.text}, ymax: {box.ymax.text}")
Make annotation on the image:
names = []
boxes = []
for crack in soup.find_all("object"):
names.append(crack.find("name").text)
box = crack.bndbox
box = {
'xmin': int(box.xmin.text),
'xmax': int(box.xmax.text),
'ymin': int(box.ymin.text),
'ymax': int(box.ymax.text)
}
boxes.append(box)
plt.figure(figsize=(10,20))
plt.imshow(img)
for name, box in zip(names, boxes):
x = [box["xmin"]] + [box["xmax"]] * 2 + [box["xmin"]] * 2
y = [box["ymin"]] * 2 + [box["ymax"]] * 2 + [box["ymin"]]
plt.plot(x, y, c="red", linewidth="2")
plt.text(box["xmax"] + 50, 0.5 * (box["ymin"] + box["ymax"]), name, c="red", fontsize=20)
plt.show()
Create directory to save annotated images:
# save_dir_name = "anno-image"
# SAVE_PATH = os.path.join(PATH, save_dir_name)
# try:
# os.mkdir(SAVE_PATH)
# except:
# all_files = os.listdir(SAVE_PATH)
# for file in all_files:
# os.remove(os.path.join(SAVE_PATH, file))
Save all annotated images:
def save_single_image(img_path, save_path, soup):
# extract crack name and position
names = []
boxes = []
for crack in soup.find_all("object"):
names.append(crack.find("name").text)
box = crack.bndbox
box = {
'xmin': int(box.xmin.text),
'xmax': int(box.xmax.text),
'ymin': int(box.ymin.text),
'ymax': int(box.ymax.text)
}
boxes.append(box)
# read image
filename = soup.filename.text
img = img_to_array(load_img(os.path.join(img_path, filename))).astype(int)
# annotate the image
fig, ax = plt.subplots(figsize=(10,20))
ax.imshow(img)
for name, box in zip(names, boxes):
x = [box["xmin"]] + [box["xmax"]] * 2 + [box["xmin"]] * 2
y = [box["ymin"]] * 2 + [box["ymax"]] * 2 + [box["ymin"]]
ax.plot(x, y, c="red", linewidth="2")
ax.text(box["xmax"] + 50, 0.5 * (box["ymin"] + box["ymax"]), name, c="red", fontsize=20)
# save image
fig.savefig(os.path.join(save_path, filename))
The following code will transform all original images into annotated images. Because this process will take a long time, we comment out this code fregment.
# %%capture
# xml_list = os.listdir(ANNO_PATH)
# xml_num = len(xml_list)
# for i, xml in enumerate(xml_list):
# with open(os.path.join(ANNO_PATH, xml)) as file:
# soup = BeautifulSoup(file.read(), 'xml')
# save_single_image(IMG_PATH, SAVE_PATH, soup)
1.2.2 Save All Labels In a List¶
filenames = []
images = []
labels = []
for xml in xml_list:
with open(os.path.join(ANNO_PATH, xml)) as file:
soup = BeautifulSoup(file.read(), 'xml')
filename = soup.filename.text
filenames.append(filename)
img = img_to_array(load_img(os.path.join(IMG_PATH, filename))).astype(int)
images.append(img)
names = set()
for crack in soup.find_all("object"):
try:
names.add(crack.find("name").text)
except:
pass
labels.append(list(names))
Let's have a look at the categories of all labels:
import pandas as pd
labels = pd.DataFrame(labels)
labels
import numpy as np
set(np.concatenate((labels[0].unique(), labels[1].unique(), labels[2].unique())).tolist())
Let's see how which images are uncracked:
null_label = labels.isnull()
null_label = null_label[0] & null_label[1] & null_label[2]
labels[null_label].index
For the first trial, we simply use binary classification, namely, cracked and uncracked.
bi_lables = np.ones(null_label.shape[0])
bi_lables[null_label] = 0
bi_lables
images[0].shape
The original image:
plt.figure(figsize=(5,12.5))
plt.imshow(images[0])
plt.show()
Resize those images to 224 $\times$ 224.
images_resized = [np.array(Image.fromarray(np.uint8(img)).resize((224,224))).astype(int) for img in images]
The output size:
images_resized[0].shape
Show a sample image.
plt.figure(figsize=(5,5))
plt.imshow(images_resized[0])
plt.show()
plt.figure(figsize=(5,12.5))
plt.imshow(images[0])
plt.show()
from scipy.stats import describe
describe(images[0].flatten())[1]
plt.hist(images[0].flatten(),256)
plt.show()
Clearly, most of the pixel values fall in the range of (100, 150), which are essentailly the grayscale values of normal road surface.
Note that since the detector used for capturing the cracks is radar-based rather CCD/CMOS-based, the pixel values does represent the real color of the road surface. In fact, pixels which are darker (approaching 0) means the distnce from the surface is farther from the detector (e.g. cracks or some barriers that prevent radar reflection at the edge of the detection range) and pixels which are lighter (approaching 255) means the distnce from the surface is shorter from the detector (e.g. road lane markings or curbs).
Due to this nature, we can multiply all the pixel values by a factor (say, 2.5) to remove all the surface information above the crack.
multiplied = np.clip(images[0] * 2.5, 0, 255).astype(int)
plt.figure(figsize=(5,12.5))
plt.imshow(multiplied)
plt.show()
Histogram for all pixels that are not 255:
plt.hist(multiplied[multiplied != 255].flatten(),256)
plt.show()
Let's we can see that only $0.5\%$ of the original information remains.
np.prod(multiplied[multiplied != 255].shape)
np.prod(images[0].shape)
np.prod(multiplied[multiplied != 255].shape) / np.prod(images[0].shape)
2.2.2 Image: 000020780000.JPG¶
This images as curb at the image edge:
plt.figure(figsize=(5,12.5))
plt.imshow(images[1])
plt.show()
This images has more right-tail values:
plt.hist(images[1].flatten(),256)
plt.show()
Multiply the pixel values:
multiplied = np.clip(images[1] * 2.5, 0, 255).astype(int)
plt.figure(figsize=(5,12.5))
plt.imshow(multiplied)
plt.show()
Histogram for all pixels that are not 255:
plt.hist(multiplied[multiplied != 255].flatten(),256)
plt.show()
$5\%$ of the original information remains.
np.prod(multiplied[multiplied != 255].shape) / np.prod(images[1].shape)
2.2.3 Image: 000001850000.JPG¶
plt.figure(figsize=(5,12.5))
plt.imshow(images[12])
plt.show()
This images has more left-tail values:
plt.hist(images[12].flatten(),256)
plt.show()
Multiply the pixel values:
multiplied = np.clip(images[12] * 2.5, 0, 255).astype(int)
plt.figure(figsize=(5,12.5))
plt.imshow(multiplied)
plt.show()
$3.8\%$ of the original information remains.
np.prod(multiplied[multiplied != 255].shape) / np.prod(images[12].shape)
2.2.4 Processing all images¶
Calculate the average proportion of non-255 pixels for all images:
prop_before = [(np.prod(img[img != 255].shape) / np.prod(img.shape)) for img in images]
np.mean(prop_before)
Perform grayscale transformation on all images:
images_gf = [np.clip(img * 2.5, 0, 255).astype(int) for img in images]
Calculate the average proportion of non-255 pixels for all images after transformation:
prop_after = [(np.prod(img[img != 255].shape) / np.prod(img.shape)) for img in images_gf]
np.mean(prop_after)
images[0].shape
We need to crop the images, the reasons include:
1. We can see from the sample images in 2.2.2 and 2.2.3 that some of dark pixel noises cannot be removed simply by grascale transformation. Fortunately, most of the noises are at the edge of the images and most of the crackes are located in the middle of the images. We can remove the noise by cut them out.
2. Most pre-trained model in transfer learning requires input images in the shape of square and of size less than 300 $\times$ 300 (usually, 224 $\times$ 224). However, our crack images is rectangular and the size is multiple of the required input. We need to cut them into small pieces.
Basically, we wish to crop the one original image into a 410 $\times$ 1230 images which containes three 410 $\times$ 410 small images as illustrated below:
ideal_shape = (1230, 410, 3)
crop_bound = {
"xmin": int((images[0].shape[1] - ideal_shape[1]) / 2),
"xmax": int((images[0].shape[1] + ideal_shape[1]) / 2 - 1),
"ymin": int((images[0].shape[0] - ideal_shape[0]) / 2),
"ymax": int((images[0].shape[0] + ideal_shape[0]) / 2 -1)
}
y_space = np.linspace(crop_bound["ymin"], crop_bound["ymax"] + 1, 4).astype(int)
plt.figure(figsize=(5,12.5))
plt.imshow(images[12])
x = [crop_bound["xmin"]] + [crop_bound["xmax"]] * 2 + [crop_bound["xmin"]] * 2
y = [crop_bound["ymin"]] * 2 + [crop_bound["ymax"]] * 2 + [crop_bound["ymin"]]
plt.plot(x, y, c="red", linewidth="2")
for i in range(1, len(y_space)):
if i != 5:
plt.plot((crop_bound["xmin"], crop_bound["xmax"]), (y_space[i], y_space[i]), color="r", linestyle="--")
plt.text((crop_bound["xmin"] + crop_bound["xmax"]) / 2 - 30, (y_space[i - 1] + y_space[i]) / 2,
i, color="orange", fontsize=40, ma="center", va="center")
plt.show()
The question is how likely for the whole cracks falls out-side of the range of this range (it seems to be ok if just a part of the crack is outside). We can check this in the XML annotation:
There are in total 303 crackes in the sample:
sample_crack_count = 0
edge_crack_count = 0
for xml in xml_list:
with open(os.path.join(ANNO_PATH, xml)) as file:
soup = BeautifulSoup(file.read(), 'xml')
objects = soup.find_all("object")
sample_crack_count += len(objects)
for crack in objects:
box = crack.bndbox
box = {
'xmin': int(box.xmin.text),
'xmax': int(box.xmax.text),
'ymin': int(box.ymin.text),
'ymax': int(box.ymax.text)
}
if (box["xmin"] > crop_bound["xmax"] or
box["xmax"] < crop_bound["xmin"] or
box["ymin"] > crop_bound["ymax"] or
box["ymax"] < crop_bound["ymin"]):
edge_crack_count += 1
sample_crack_count
Of which 5 crackes will be cut out.
edge_crack_count
The cracks cut out accounts for roughly 2% of the total crack. It is reasonably accetable.
edge_crack_count / sample_crack_count
To decide which crop size is most suitable, we can plot the wide of image after being cropped (520 to 310) against the proprotion of crack that will be cut out, as well as the length of image after bing cropped (1250 to 1000) against the proprotion of crack that is cut out.
wide = np.arange(310, 521)
length = np.arange(1000, 1250)
crop_bounds_wide = [{
"xmin": int((images[0].shape[1] - i) / 2),
"xmax": int((images[0].shape[1] + i) / 2 - 1)
} for i in wide]
crop_bounds_length = [{
"ymin": int((images[0].shape[0] - i) / 2),
"ymax": int((images[0].shape[0] + i) / 2 - 1)
} for i in length]
sample_crack_count = 0
edge_crack_count_wide = np.zeros(len(wide))
edge_crack_count_length = np.zeros(len(length))
for xml in xml_list:
with open(os.path.join(ANNO_PATH, xml)) as file:
soup = BeautifulSoup(file.read(), 'xml')
objects = soup.find_all("object")
sample_crack_count += len(objects)
for crack in objects:
box = crack.bndbox
box = {
'xmin': int(box.xmin.text),
'xmax': int(box.xmax.text),
'ymin': int(box.ymin.text),
'ymax': int(box.ymax.text)
}
for idx, crop_bound in enumerate(crop_bounds_wide):
if (box["xmin"] > crop_bound["xmax"] or
box["xmax"] < crop_bound["xmin"]):
edge_crack_count_wide[idx] += 1
for idx, crop_bound in enumerate(crop_bounds_length):
if (box["ymin"] > crop_bound["ymax"] or
box["ymax"] < crop_bound["ymin"]):
edge_crack_count_length[idx] += 1
plt.figure()
plt.plot(wide, edge_crack_count_wide / sample_crack_count)
plt.xlabel('Wide of images after being cropped')
plt.ylabel('Proportion of crack lost')
plt.show()
plt.figure()
plt.plot(length, edge_crack_count_length / sample_crack_count)
plt.xlabel('Length of images after being cropped')
plt.ylabel('Proportion of crack lost')
plt.show()
We can see the any wide > 400 and any length > 1100 is OK for the proportion of crack to be cut out stays below roughly 2%.
So, our choice of 420 $\times$ 1240 is acceptable.
2.3.2 Crop all Images¶
Crop all original images:
ideal_shape = (1230, 410, 3)
crop_bound = {
"xmin": int((images[0].shape[1] - ideal_shape[1]) / 2),
"xmax": int((images[0].shape[1] + ideal_shape[1]) / 2 - 1),
"ymin": int((images[0].shape[0] - ideal_shape[0]) / 2),
"ymax": int((images[0].shape[0] + ideal_shape[0]) / 2 -1)
}
images_cropped = [img[crop_bound["ymin"]:crop_bound["ymax"] + 1, crop_bound["xmin"]:crop_bound["xmax"] + 1] for img in images]
images_cropped[0].shape
We can see the noises at the edge in original sample image from 2.2.3 has been removed
plt.figure(figsize=(5, 6.5))
plt.subplot(121)
plt.imshow(images[12])
plt.title("Original Image")
plt.subplot(122)
plt.imshow(images_cropped[12])
plt.title("Cropped Image")
plt.show()
Then we can split the cropped image into 3 sqaure 410 $\times$ 410 images:
y_space = np.linspace(0, ideal_shape[0], 4).astype(int)
y_space[-1] += 1
images_split = [[img[y_space[i]:y_space[i + 1],] for i in range(len(y_space) - 1)] for img in images_cropped]
Check whether the images are corretly split.
plt.figure(figsize=(8, 12))
plt.subplot(1,2,1)
plt.imshow(images_cropped[12])
plt.title("Cropped Image")
for i in range(1, 4):
plt.subplot(3,2,i * 2)
plt.imshow(images_split[12][i - 1])
plt.title(f"Splitted Image {i}")
plt.show()
We can overlay the 3 images by taking the minimum of each pixel:
images_overlay = [np.minimum(np.minimum(img[0], img[1]), img[2]) for img in images_split]
plt.figure(figsize=(5, 5))
plt.imshow(images_overlay[12])
plt.title("Overlay of the 3 splitted image")
plt.show()
images_overlay[0].shape
Then we can resize the image to 224 $\times$ 224.
images_overlay_resized = [np.array(Image.fromarray(np.uint8(img)).resize((224,224))).astype(int) for img in images_overlay]
images_overlay_resized[0].shape
Have a look at the exmaple:
plt.figure(figsize=(5, 5))
plt.imshow(images_overlay_resized[12])
plt.title("Resized overlay of the 3 splitted image")
plt.show()
3. Extract Coordinate Information¶
long_deg, long_min, long_sec, lat_deg, lat_min, lat_sec = [], [], [], [], [], []
for filename in filenames:
img = Image.open(os.path.join(IMG_PATH, filename))
meta = list(img._getexif().items())[0][1]
long_deg.append(meta[2][0][0])
long_min.append(meta[2][1][0])
long_sec.append(meta[2][2][0] / meta[2][2][1])
lat_deg.append(meta[4][0][0])
lat_min.append(meta[4][1][0])
lat_sec.append(meta[4][2][0] / meta[2][2][1])
coord = pd.DataFrame({
"long_deg": long_deg,
"long_min": long_min,
"long_sec": long_sec,
"lat_deg": lat_deg,
"lat_min": lat_min,
"lat_sec": lat_sec
})
coord